package main

import (
	"cms/libs"
	"cms/model"
	"crypto/md5"
	"crypto/tls"
	"encoding/hex"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"github.com/axgle/mahonia"
	"github.com/gocolly/colly"
	"github.com/gocolly/colly/extensions"
	"github.com/jinzhu/gorm"
	config "github.com/spf13/viper"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"time"
)

var ExistsUrlMap sync.Map
var wg sync.WaitGroup
var proxies []*url.URL = []*url.URL{}
var DefaultClient = &http.Client{
	Timeout: 3 * time.Second,
	Transport: &http.Transport{ //解决x509: certificate signed by unknown authority
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		Proxy:           http.ProxyFromEnvironment,
		DialContext: (&net.Dialer{
			Timeout:   3 * time.Second,
			KeepAlive: 10 * time.Second,
			DualStack: true,
		}).DialContext,
		//MaxIdleConns:          100,
		IdleConnTimeout:       90 * time.Second,
		TLSHandshakeTimeout:   10 * time.Second,
		ExpectContinueTimeout: 1 * time.Second,
	},
	CheckRedirect: func(req *http.Request, via []*http.Request) error { //禁止重定向
		return http.ErrUseLastResponse
	},
}

type Nav struct {
	gorm.Model
	CategoryId   uint   `gorm:"column:category_id" json:"category_id" form:"category_id"`
	ShowPosition uint   `gorm:"column:show_position" json:"show_position" form:"show_position"`
	URL          string `gorm:"column:url" json:"url" form:"url"`
	Title        string `gorm:"column:title" json:"title" form:"title"`
	Desc         string `gorm:"column:desc" json:"desc" form:"desc"`
	Keywords     string `gorm:"column:keywords" json:"keywords" form:"keywords"`
	From         string `gorm:"column:from" json:"from" form:"from"`
	Sort         uint   `gorm:"column:sort" json:"sort" form:"sort"`
	Hits         uint   `gorm:"column:hits" json:"hits" form:"hits"`
	Image        string `gorm:"column:image" json:"image" form:"image"`
	Recommend    string `gorm:"column:recommend" json:"recommend" form:"recommend"`
}

type NavHistory struct {
	Key string `gorm:"column:key" json:"key" form:"key"`
}

func init() {
	config.AddConfigPath("./configs")
	config.SetConfigName("mysql")
	if err := config.ReadInConfig(); err != nil {
		panic("Error reading config file, " + err.Error())
	}

	dbConfig := libs.DbConfig{
		config.GetString("default.host"),
		config.GetString("default.port"),
		config.GetString("default.database"),
		config.GetString("default.user"),
		config.GetString("default.password"),
		config.GetString("default.charset"),
		config.GetInt("default.MaxIdleConns"),
		config.GetInt("default.MaxOpenConns"),
	}
	libs.DB = dbConfig.InitDB()
	if config.GetBool("default.sql_log") {
		libs.DB.LogMode(true)
	}
}

//自定义 网址，深度，代理池
func main() {
	urlList := []string{
		"http://www.gongyedh.com/",
		"http://www.33591.com/",
	}
	proxiesList := []string{
		//"127.0.0.1:8080",
		//"127.0.0.1:8081",
	}

	var setUrlList model.Setting
	var setProxyList model.Setting
	libs.DB.Table("setting").Where("`key` = ?", "SpiderUrl").First(&setUrlList)
	if setUrlList.Val == "" {
		return
	}
	urlList = strings.Split(setUrlList.Val, "\r\n")
	libs.DB.Table("setting").Where("`key` = ?", "SpiderProxy").First(&setProxyList)
	if setProxyList.Val != "" {
		proxiesList = strings.Split(setProxyList.Val, "\r\n")
	}

	for _, v := range proxiesList {
		proxies = append(proxies, &url.URL{Host: v})
	}
	spider(urlList)
	os.Exit(1)
	return
}

func spider(urlList []string) {
	AllowedDomains := []string{}
	SpiderUrls := []string{}
	for _, v := range urlList {
		parseUrl := strings.Split(v, "|")
		if len(parseUrl) == 2 {
			baseUrlParse, err := url.Parse(parseUrl[0])
			if err == nil {
				SpiderUrls = append(SpiderUrls, parseUrl[0])
				AllowedDomains = append(AllowedDomains, baseUrlParse.Host)
			}
		}
	}
	c := colly.NewCollector(
		colly.MaxDepth(2),
		colly.CacheDir("./cache"),
		colly.Async(true),
		//colly.AllowURLRevisit(),
		//colly.Debugger(&debug.LogDebugger{}),
	)
	c.AllowedDomains = AllowedDomains
	if len(proxies) > 0 {
		c.SetProxyFunc(randomProxySwitcher)
	}

	c.WithTransport(&http.Transport{
		Proxy: http.ProxyFromEnvironment,
		DialContext: (&net.Dialer{
			Timeout:   5 * time.Second,
			KeepAlive: 10 * time.Second,
			DualStack: true,
		}).DialContext,
		//MaxIdleConns:          0,
		IdleConnTimeout:       90 * time.Second,
		ResponseHeaderTimeout: 10 * time.Second,
		ExpectContinueTimeout: 1 * time.Second,
		DisableKeepAlives:     true,
	})

	extensions.RandomUserAgent(c)
	extensions.Referer(c)

	_ = c.Limit(&colly.LimitRule{
		DomainGlob: "*",
		//Parallelism: 2, //并发
		//RandomDelay: 5 * time.Second,
	})

	c.OnHTML("a[href]", func(e *colly.HTMLElement) {
		link := e.Attr("href")
		err := c.Visit(e.Request.AbsoluteURL(link))
		if err == nil {
			return
		}
		u, err := url.Parse(link)
		if err != nil {
			return
		}

		for _, v := range AllowedDomains {
			if u.Host == v {
				return
			}
		}

		if len(u.Scheme) < 4 {
			return
		}
		if u.Scheme[0:4] != "http" {
			return
		}
		if u.Host[0:3] == "www" || strings.Count(u.Host, ".") == 1 {
			if _, ok := ExistsUrlMap.Load(u.Host); !ok {
				ExistsUrlMap.Store(u.Host, true)
				RunData(u, e)
			}
		}

	})

	c.OnRequest(func(r *colly.Request) {
		//fmt.Println("Visiting", r.URL.String())
	})

	for _, v := range SpiderUrls {
		_ = c.Visit(v)
	}

	c.Wait()
	return
}

func RunData(u *url.URL, e *colly.HTMLElement) {
	return
	fmt.Println(u.Scheme + "://" + u.Host)
	res, err := DefaultClient.Get(u.Scheme + "://" + u.Host)
	if err != nil {
		fmt.Println(err.Error())
		return
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		return
	}
	document, err := goquery.NewDocumentFromReader(res.Body)
	if err != nil {
		return
	}
	title := ""
	keywords := ""
	description := ""
	title = document.Find("title").Eq(0).Text()
	if title == "" {
		return
	}
	document.Find("meta").Each(func(i int, s *goquery.Selection) {
		if name, _ := s.Attr("name"); strings.EqualFold(name, "keywords") {
			keywords, _ = s.Attr("content")
		}
		if name, _ := s.Attr("name"); strings.EqualFold(name, "description") {
			description, _ = s.Attr("content")
		}
	})

	Nav := Nav{
		ShowPosition: 0,
		CategoryId:   0,
		Title:        ConvertToString(title),
		URL:          u.Scheme + "://" + u.Host,
		Desc:         ConvertToString(description),
		Keywords:     ConvertToString(keywords),
		From:         "采集",
		Sort:         0,
		Hits:         0,
		Image:        "",
		Recommend:    "",
	}

	historyKey := md5V(Nav.URL)
	if libs.DB.Table("nav_history").Where("`key` = ?", historyKey).First(&NavHistory{}).RecordNotFound() {
		err = libs.DB.Table("nav_history").Create(&NavHistory{Key: historyKey}).Error
		if err != nil {
			return
		}
		_ = libs.DB.Table("nav").Create(&Nav).Error
	}
	//fmt.Println(Nav)
}

func randomProxySwitcher(_ *http.Request) (*url.URL, error) {
	return proxies[rand.Intn(len(proxies))], nil
}

func md5V(str string) string {
	h := md5.New()
	h.Write([]byte(str))
	return hex.EncodeToString(h.Sum(nil))
}

func ConvertToString(src string) string {
	if isUtf8([]byte(src)) {
		return src
	}
	srcCoder := mahonia.NewDecoder("gbk")
	srcResult := srcCoder.ConvertString(src)
	tagCoder := mahonia.NewDecoder("utf-8")
	_, cdata, _ := tagCoder.Translate([]byte(srcResult), true)
	result := string(cdata)
	return result
}

func preNUm(data byte) int {
	str := fmt.Sprintf("%b", data)
	var i int = 0
	for i < len(str) {
		if str[i] != '1' {
			break
		}
		i++
	}
	return i
}

func isUtf8(data []byte) bool {
	for i := 0; i < len(data); {
		if data[i]&0x80 == 0x00 {
			// 0XXX_XXXX
			i++
			continue
		} else if num := preNUm(data[i]); num > 2 {
			// 110X_XXXX 10XX_XXXX
			// 1110_XXXX 10XX_XXXX 10XX_XXXX
			// 1111_0XXX 10XX_XXXX 10XX_XXXX 10XX_XXXX
			// 1111_10XX 10XX_XXXX 10XX_XXXX 10XX_XXXX 10XX_XXXX
			// 1111_110X 10XX_XXXX 10XX_XXXX 10XX_XXXX 10XX_XXXX 10XX_XXXX
			// preNUm() 返回首个字节的8个bits中首个0bit前面1bit的个数，该数量也是该字符所使用的字节数
			i++
			for j := 0; j < num-1; j++ {
				//判断后面的 num - 1 个字节是不是都是10开头
				if data[i]&0xc0 != 0x80 {
					return false
				}
				i++
			}
		} else {
			//其他情况说明不是utf-8
			return false
		}
	}
	return true
}
